
#from IPython.core.display import display, HTML
#display(HTML("<style>.container { width:100% !important; }</style>"))

import pandas as pd
import numpy as np
import sklearn as sk
from datetime import timedelta
from sklearn.preprocessing import LabelEncoder
from sklearn.impute import KNNImputer
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
import scipy as sp
import math
import statistics
from pandas_profiling import ProfileReport
import lightgbm as lgb
import optuna as op
from optuna.visualization import plot_optimization_history
from optuna.visualization import plot_intermediate_values
from optuna.visualization import plot_parallel_coordinate
from optuna.visualization import plot_contour
from optuna.visualization import plot_slice
from optuna.visualization import plot_param_importances
import pickle
import plotly
import nbconvert


trainingdata = pd.read_csv('data/transactions_train.csv')
articlesdata = pd.read_csv('data/articles.csv')
customersdata = pd.read_csv('data/customers.csv')


trainingdata['t_dat'] = pd.to_datetime(trainingdata['t_dat'], yearfirst = True)
print(trainingdata['t_dat'].max())
max_date = trainingdata['t_dat'].min() + timedelta(days=140)
min_date = trainingdata['t_dat'].max() - timedelta(days=70)
print(min_date)
print(max_date)
trainingdata=trainingdata.loc[trainingdata['t_dat'] > min_date]

trainingdata['days_from_start'] = (trainingdata['t_dat'] - trainingdata['t_dat'].min()).dt.days.astype('int16')
trainingdata['bucket'] = (trainingdata['days_from_start'] / 7).apply(np.floor)

2020-09-22 00:00:00
2020-07-14 00:00:00
2019-02-07 00:00:00


#merged_set = trainingdata.merge(articlesdata, left_on='article_id', right_on='article_id',
#          suffixes=('_left', '_right'))
merged_set =trainingdata.merge(customersdata, left_on='customer_id', right_on='customer_id',
          suffixes=('_left', '_right'))
del trainingdata


profile = ProfileReport(merged_set, title="Pandas Profiling Report", explorative=True)
profile.to_notebook_iframe()
profile.to_file("eda_analysis/eda_part_1.html")

Summarize dataset:   0%|          | 0/5 [00:00<?, ?it/s]

Generate report structure:   0%|          | 0/1 [00:00<?, ?it/s]

Render HTML:   0%|          | 0/1 [00:00<?, ?it/s]

Export report to file:   0%|          | 0/1 [00:00<?, ?it/s]


x = merged_set.drop(columns=['t_dat','article_id','price','sales_channel_id','days_from_start','bucket','FN','Active'])
x.drop_duplicates(inplace=True)
x.reset_index(inplace=True)
x.head()


#pivoted_merged_data = merged_set.pivot_table(index='customer_id', columns='t_dat', values='article_id', aggfunc=lambda x: ' '.join(x))
pivoted_merged_data = merged_set.pivot_table(index=['customer_id'],
                             columns=['bucket'],
                             values=['article_id'],
                             aggfunc=lambda x: ' '.join(str(v) for v in x))
pivoted_merged_data.head()


training_data_set = x.join(pivoted_merged_data,on=['customer_id'], how='inner')
training_data_set.drop(columns=['index'], inplace = True)
training_data_set = training_data_set.loc[training_data_set['age'] < 30]
training_data_set.head()

d:\soprisanalytics\kaggle\dtsa-5509-supervised-learning-final-project\venv\lib\site-packages\pandas\core\frame.py:9130: FutureWarning: merging between different levels is deprecated and will be removed in a future version. (1 levels on the left,2 on the right)
  return merge(


#del x
#del pivoted_merged_data
#training_data_set.rename(columns={"('article_id', 0.0)": "0", "('article_id', 1.0)": "1"}, errors="raise", inplace = True)
#training_data_set["('article_id', 0.0)"].head()
i = 0
for each in range(5,len(training_data_set.columns)):
    training_data_set.columns.values[each] = f'{i}_week'
    i+=1

for each in range(0,4):  
    column_name = training_data_set.columns.tolist()[each]
    if training_data_set[column_name].dtypes == 'object':
        #training_data_set[column_name]=training_data_set[column_name].replace(np.nan, 'No')
        training_data_set.iloc[:,each].replace(np.nan, 'No',inplace = True)
    else:
        training_data_set.iloc[:,each].replace(np.nan, statistics.mode(training_data_set[column_name]),inplace = True)
        #training_data_set[column_name]=training_data_set[column_name].replace(np.nan, statistics.mode(training_data_set[column_name]))
i=0
for each in range(5,len(training_data_set.columns)):
    column_name = training_data_set.columns.tolist()[each]
    training_data_set.iloc[:,each].replace(np.nan, '1',inplace = True)                   
    i+=1
training_data_set = training_data_set.iloc[:500000].sort_values(by=['age'])
training_data_set.head()


customer_id_encoding = LabelEncoder()
training_data_set['customer_ids'] = customer_id_encoding.fit_transform(training_data_set['customer_id'])

club_member_status_encoding = LabelEncoder()
training_data_set['club_member_statuses'] = club_member_status_encoding.fit_transform(training_data_set['club_member_status'])

fashion_news_frequency_encoding = LabelEncoder()
training_data_set['fashion_news_frequencys'] = fashion_news_frequency_encoding.fit_transform(training_data_set['fashion_news_frequency'])

postal_code_encoding = LabelEncoder()
training_data_set['postal_codes'] = postal_code_encoding.fit_transform(training_data_set['postal_code'])

X = training_data_set.drop(columns=['customer_id','club_member_status','fashion_news_frequency','postal_code'])

column_to_move = X.pop("customer_ids")
X.insert(0, "customer_id", column_to_move)

column_to_move = X.pop("club_member_statuses")
X.insert(0, "club_member_status", column_to_move)

column_to_move = X.pop("fashion_news_frequencys")
X.insert(0, "fashion_news_frequency", column_to_move)

column_to_move = X.pop("postal_codes")
X.insert(0, "postal_codes", column_to_move)


profile = ProfileReport(X, title="Pandas Profiling Report", explorative=True)
profile.to_notebook_iframe()
profile.to_file("eda_analysis/eda_part_2.html")

Summarize dataset:   0%|          | 0/5 [00:00<?, ?it/s]

Generate report structure:   0%|          | 0/1 [00:00<?, ?it/s]

Render HTML:   0%|          | 0/1 [00:00<?, ?it/s]

Export report to file:   0%|          | 0/1 [00:00<?, ?it/s]


class neighborObjective(object):
    def __init__(self,X):
        i = 0
        labels =[]
        X = X.sample(5000)
        for each in range(5,len(X.columns)):
            labels.extend(X.iloc[:,each].tolist())
            i+=1
        labels = list(set(labels))
        encoder = LabelEncoder()
        fit = encoder.fit(labels)
        self.null = fit.transform(['1'])
        i = 0
        for each in range(5,len(X.columns)):
            original_column = f'{i}_week'
            new_column = f'{i}_weeks'
            X.loc[:, (f'{i}_weeks')] = fit.transform(X[f'{i}_week'])
            #X[f'{i}_weeks']=X[f'{i}_weeks'].replace(0, np.nan)
            X = X.drop(columns=[f'{i}_week'])
            i+=1
        X_raw = X
        N = len(X_raw.columns)
        last_column = N-1
        second_to_last_column = N-2
        X = X_raw.iloc[: , 0:second_to_last_column]
        y = X_raw.iloc[: ,last_column]
        self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(X, y, test_size=0.20)
        
    def knn(self,trial):
        n = trial.suggest_int("neighbors", 2, 20, log=True)
        neigh = KNeighborsClassifier(n_neighbors=n)
        neigh.fit(self.X_train,self.y_train)
        neighbors = pd.DataFrame(neigh.kneighbors(return_distance=False))
        prediction = pd.DataFrame(neigh.predict(self.X_test))     
        return neighbors, prediction
    
    def lightgbm(self, trial, neighbors, prediction):
        X_train = self.X_train.join(neighbors)
        X_test = self.X_test.join(neighbors)
        X_train = self.X_train.join(prediction)
        X_test = self.X_test.join(prediction)       
        
        y_train = self.y_train
        y_test = self.y_test
        y_train[self.y_train == self.null[0]] = 1  
        y_train[self.y_train != 1] = -1
        y_test[self.y_test == self.null[0]] = 1         
        y_test[self.y_test != 1] = -1
        dtrain = lgb.Dataset(X_train, label=y_train)
        dvalid = lgb.Dataset(X_test, label=y_test)
        param = {
            "objective": "binary",
            "metric": "binary_logloss",
            "verbosity": 0,
            "boosting_type": "gbdt",
            "num_leaves": trial.suggest_int("num_leaves", 2, 256),
            "feature_fraction": trial.suggest_float("feature_fraction", 0.4, 1.0),
            "bagging_fraction": trial.suggest_float("bagging_fraction", 0.4, 1.0),
            "bagging_freq": trial.suggest_int("bagging_freq", 1, 7),
            "min_child_samples": trial.suggest_int("min_child_samples", 5, 100),
            "early_stopping_round":1
        }

        gbm = lgb.train(param, dtrain, valid_sets=dvalid)
        preds = gbm.predict(X_test)
        pred_labels = np.rint(preds)
        accuracy = sk.metrics.f1_score(y_test, pred_labels, average='macro')
        return accuracy
    
    def __call__(self, trial):
        neighbors, prediction = self.knn(trial)
        accuracy = self.lightgbm(trial, neighbors, prediction)
        return accuracy
    
if __name__ == "__main__":
    print(np.unique(X['age'],return_counts=True))

    best_models=[]
    age_iteration = 5
    age_range = range(int(min(X['age'])),int(max(X['age'])),age_iteration)
    plots = []
    for young_age in age_range:
        print(young_age)
        old_age = young_age+age_iteration
        print(old_age)
        age_study = X.loc[(X['age'] >= young_age) & (X['age'] <= old_age)]
        print(len(age_study))
        
        age_study = age_study.iloc[:10000]
        
        study = op.create_study(direction="maximize")
        accuracy = study.optimize(neighborObjective(age_study), n_trials=20)
        
        fig = plot_optimization_history(study)
        fig.write_html(f"optimization_graphs/{young_age}_optimization_history.html")
        plots.append(fig)
        
        fig = plot_parallel_coordinate(study)
        fig.write_html(f"optimization_graphs/{young_age}_parallel_coordinate.html")
        plots.append(fig)
        
        fig = plot_contour(study)
        fig.write_html(f"optimization_graphs/{young_age}_contour.html")
        plots.append(fig)
        
        fig = plot_slice(study)
        fig.write_html(f"optimization_graphs/{young_age}_plot_slice.html")
        plots.append(fig)
        
        #plot_param_importances(study)
        
        best_models.append([study.best_params,young_age,age_iteration,accuracy])
    print(study.best_trial)

[I 2022-03-01 17:31:18,855] A new study created in memory with name: no-name-c1013a18-b801-4825-8a8d-e0dac321cfd4

(array([16., 17., 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28.,
       29.]), array([   65,  3942,  7539, 12494, 18702, 22824, 18630, 20515, 21089,
       20493, 19967, 17602, 15538, 14031], dtype=int64))
16
21
65566

[I 2022-03-01 17:31:19,373] Trial 0 finished with value: 0.4617868675995694 and parameters: {'neighbors': 3, 'num_leaves': 173, 'feature_fraction': 0.5186323020532635, 'bagging_fraction': 0.6163552066220521, 'bagging_freq': 6, 'min_child_samples': 31}. Best is trial 0 with value: 0.4617868675995694.

[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.000704 seconds.
You can set `force_row_wise=true` to remove the overhead.
And if memory is not enough, you can set `force_col_wise=true`.
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[1]	valid_0's binary_logloss: 0.406331
Training until validation scores don't improve for 1 rounds
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[2]	valid_0's binary_logloss: 0.403762
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[3]	valid_0's binary_logloss: 0.399445
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[4]	valid_0's binary_logloss: 0.397679
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[5]	valid_0's binary_logloss: 0.395471
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[6]	valid_0's binary_logloss: 0.393805
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[7]	valid_0's binary_logloss: 0.39275
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[8]	valid_0's binary_logloss: 0.389347
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[9]	valid_0's binary_logloss: 0.388336
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[10]	valid_0's binary_logloss: 0.387793
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[11]	valid_0's binary_logloss: 0.388126
Early stopping, best iteration is:
[10]	valid_0's binary_logloss: 0.387793
[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.000380 seconds.
You can set `force_row_wise=true` to remove the overhead.
And if memory is not enough, you can set `force_col_wise=true`.
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[1]	valid_0's binary_logloss: 0.400834
Training until validation scores don't improve for 1 rounds
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[2]	valid_0's binary_logloss: 0.395641
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[3]	valid_0's binary_logloss: 0.39098
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[4]	valid_0's binary_logloss: 0.38318
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[5]	valid_0's binary_logloss: 0.377515
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[6]	valid_0's binary_logloss: 0.36907
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf

[I 2022-03-01 17:31:19,538] Trial 1 finished with value: 0.30785907859078593 and parameters: {'neighbors': 7, 'num_leaves': 107, 'feature_fraction': 0.7137972448961027, 'bagging_fraction': 0.7648585770407605, 'bagging_freq': 2, 'min_child_samples': 70}. Best is trial 0 with value: 0.4617868675995694.
[I 2022-03-01 17:31:19,672] Trial 2 finished with value: 0.4617868675995694 and parameters: {'neighbors': 7, 'num_leaves': 185, 'feature_fraction': 0.4403344560293559, 'bagging_fraction': 0.6689355433676443, 'bagging_freq': 2, 'min_child_samples': 46}. Best is trial 0 with value: 0.4617868675995694.

[7]	valid_0's binary_logloss: 0.367343
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[8]	valid_0's binary_logloss: 0.359588
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[9]	valid_0's binary_logloss: 0.35673
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[10]	valid_0's binary_logloss: 0.353932
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[11]	valid_0's binary_logloss: 0.353246
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[12]	valid_0's binary_logloss: 0.35134
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[13]	valid_0's binary_logloss: 0.349937
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[14]	valid_0's binary_logloss: 0.348179
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[15]	valid_0's binary_logloss: 0.347072
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[16]	valid_0's binary_logloss: 0.346204
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[17]	valid_0's binary_logloss: 0.345858
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[18]	valid_0's binary_logloss: 0.343842
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[19]	valid_0's binary_logloss: 0.342984
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[20]	valid_0's binary_logloss: 0.34193
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[21]	valid_0's binary_logloss: 0.339709
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[22]	valid_0's binary_logloss: 0.337492
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[23]	valid_0's binary_logloss: 0.337643
Early stopping, best iteration is:
[22]	valid_0's binary_logloss: 0.337492
[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.000386 seconds.
You can set `force_row_wise=true` to remove the overhead.
And if memory is not enough, you can set `force_col_wise=true`.
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[1]	valid_0's binary_logloss: 0.406373
Training until validation scores don't improve for 1 rounds
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[2]	valid_0's binary_logloss: 0.404237
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[3]	valid_0's binary_logloss: 0.404417
Early stopping, best iteration is:
[2]	valid_0's binary_logloss: 0.404237

[I 2022-03-01 17:31:19,820] Trial 3 finished with value: 0.4617868675995694 and parameters: {'neighbors': 12, 'num_leaves': 20, 'feature_fraction': 0.7856558769631206, 'bagging_fraction': 0.4258870980509263, 'bagging_freq': 2, 'min_child_samples': 43}. Best is trial 0 with value: 0.4617868675995694.

[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.000530 seconds.
You can set `force_row_wise=true` to remove the overhead.
And if memory is not enough, you can set `force_col_wise=true`.
[1]	valid_0's binary_logloss: 0.400108
Training until validation scores don't improve for 1 rounds
[2]	valid_0's binary_logloss: 0.39449
[3]	valid_0's binary_logloss: 0.389125
[4]	valid_0's binary_logloss: 0.380948
[5]	valid_0's binary_logloss: 0.37636
[6]	valid_0's binary_logloss: 0.368781
[7]	valid_0's binary_logloss: 0.367636
[8]	valid_0's binary_logloss: 0.360134
[9]	valid_0's binary_logloss: 0.358628
[10]	valid_0's binary_logloss: 0.35562
[11]	valid_0's binary_logloss: 0.35449
[12]	valid_0's binary_logloss: 0.352559
[13]	valid_0's binary_logloss: 0.350806
[14]	valid_0's binary_logloss: 0.34869
[15]	valid_0's binary_logloss: 0.34817
[16]	valid_0's binary_logloss: 0.348291
Early stopping, best iteration is:
[15]	valid_0's binary_logloss: 0.34817
[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.000513 seconds.
You can set `force_row_wise=true` to remove the overhead.
And if memory is not enough, you can set `force_col_wise=true`.
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[1]	valid_0's binary_logloss: 0.404419
Training until validation scores don't improve for 1 rounds
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[2]	valid_0's binary_logloss: 0.402146
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[3]	valid_0's binary_logloss: 0.39949
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[4]	valid_0's binary_logloss: 0.397177
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[5]	valid_0's binary_logloss: 0.39305
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[6]	valid_0's binary_logloss: 0.388657
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[7]	valid_0's binary_logloss: 0.387302
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[8]	valid_0's binary_logloss: 0.385976
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[9]	valid_0's binary_logloss: 0.381427
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[10]	valid_0's binary_logloss: 0.379976
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[11]	valid_0's binary_logloss: 0.379442
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[12]	valid_0's binary_logloss: 0.376329

[I 2022-03-01 17:31:19,997] Trial 4 finished with value: 0.4617868675995694 and parameters: {'neighbors': 8, 'num_leaves': 256, 'feature_fraction': 0.5151810779827115, 'bagging_fraction': 0.8225336932989005, 'bagging_freq': 2, 'min_child_samples': 25}. Best is trial 0 with value: 0.4617868675995694.
[I 2022-03-01 17:31:20,159] Trial 5 finished with value: 0.4617868675995694 and parameters: {'neighbors': 16, 'num_leaves': 40, 'feature_fraction': 0.45894312808773186, 'bagging_fraction': 0.477543533951297, 'bagging_freq': 3, 'min_child_samples': 73}. Best is trial 0 with value: 0.4617868675995694.

[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[13]	valid_0's binary_logloss: 0.374671
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[14]	valid_0's binary_logloss: 0.372128
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[15]	valid_0's binary_logloss: 0.372369
Early stopping, best iteration is:
[14]	valid_0's binary_logloss: 0.372128
[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.000354 seconds.
You can set `force_row_wise=true` to remove the overhead.
And if memory is not enough, you can set `force_col_wise=true`.
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[1]	valid_0's binary_logloss: 0.405265
Training until validation scores don't improve for 1 rounds
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[2]	valid_0's binary_logloss: 0.403636
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[3]	valid_0's binary_logloss: 0.399178
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[4]	valid_0's binary_logloss: 0.398328
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[5]	valid_0's binary_logloss: 0.395179
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[6]	valid_0's binary_logloss: 0.393454
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[7]	valid_0's binary_logloss: 0.392806
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[8]	valid_0's binary_logloss: 0.388345
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[9]	valid_0's binary_logloss: 0.3877
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[10]	valid_0's binary_logloss: 0.386678
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[11]	valid_0's binary_logloss: 0.385147
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[12]	valid_0's binary_logloss: 0.384199
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[13]	valid_0's binary_logloss: 0.381407
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[14]	valid_0's binary_logloss: 0.379853
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[15]	valid_0's binary_logloss: 0.378516
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[16]	valid_0's binary_logloss: 0.376737
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[17]	valid_0's binary_logloss: 0.376379
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[18]	valid_0's binary_logloss: 0.374028
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[19]	valid_0's binary_logloss: 0.371069
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[20]	valid_0's binary_logloss: 0.369023
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[21]	valid_0's binary_logloss: 0.365865
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[22]	valid_0's binary_logloss: 0.363229
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[23]	valid_0's binary_logloss: 0.362299
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[24]	valid_0's binary_logloss: 0.362805
Early stopping, best iteration is:
[23]	valid_0's binary_logloss: 0.362299


for fig in plots:
    fig.show()


print(best_models)

[[{'neighbors': 3, 'num_leaves': 173, 'feature_fraction': 0.5186323020532635, 'bagging_fraction': 0.6163552066220521, 'bagging_freq': 6, 'min_child_samples': 31}, 16, 5, None], [{'neighbors': 2, 'num_leaves': 115, 'feature_fraction': 0.8414673863714155, 'bagging_fraction': 0.5192146843669733, 'bagging_freq': 7, 'min_child_samples': 69}, 21, 5, None], [{'neighbors': 2, 'num_leaves': 130, 'feature_fraction': 0.7006697050976558, 'bagging_fraction': 0.9226721145541669, 'bagging_freq': 5, 'min_child_samples': 98}, 26, 5, None]]


class train(object):
    def __init__(self,X,model_params):
        self.model_params = model_params
        i = 0
        labels =[]
        for each in range(5,len(X.columns)):
            labels.extend(X.iloc[:,each].tolist())
            i+=1
        labels = list(set(labels))
        encoder = LabelEncoder()
        fit = encoder.fit(labels)
        self.fit = fit
        self.null = fit.transform(['1'])
        i = 0
        for each in range(5,len(X.columns)):
            original_column = f'{i}_week'
            new_column = f'{i}_weeks'
            X.loc[:, (f'{i}_weeks')] = fit.transform(X[original_column]).astype(int)
            #X[f'{i}_weeks']=X[f'{i}_weeks'].replace(0, np.nan)
            X = X.drop(columns=[f'{i}_week'])
            i+=1
        #X[f'{i+1}_week']='1'
        N = len(X.columns)
        last_column = N-1
        second_to_last_column = N-2
        self.X = X.iloc[: , 0:second_to_last_column]
        self.y = X.iloc[: ,last_column]     
        
        X = X.drop(columns=['0_weeks'])
        X = X.drop(columns=['1_weeks'])
        X = X.drop(columns=['2_weeks'])
        
        d = 3
        i = 3
        for each in range(5,len(X.columns)):
            original_column = f'{i}_weeks'
            new_column = f'{i-d}_weeks'
            X[new_column] = X[original_column].astype(int)
            X = X.drop(columns=[original_column])
            i+=1
        print(self.null)
        X[f'{i-d}_weeks'] = self.null[0]
        N = len(X.columns)
        last_column = N
        self.X_pred = X.iloc[: , 0:last_column]
        
    def knn(self):
        neigh = KNeighborsClassifier(n_neighbors=self.model_params['neighbors'])
        neigh.fit(self.X,self.y)
        neighbors = pd.DataFrame(neigh.kneighbors(return_distance=False))
        prediction = pd.DataFrame(neigh.predict(self.X))       
        return neigh, neighbors, prediction
    
    def data(self, X_data, neighbors, prediction):
        X = X_data.join(neighbors)
        X = X_data.join(prediction)       
        self.X = X
        
        y = self.y
        y[self.y == self.null[0]] = 1  
        y[self.y != 1] = -1
        return X, y
    
    def lightgbm(self,neighbors, prediction):
        X, y = self.data(self.X,neighbors,prediction)
        dtrain = lgb.Dataset(X, label=y)
        param = {
            "objective": "binary",
            "metric": "binary_logloss",
            "verbosity": 1,
            "boosting_type": "gbdt",
            "num_leaves": self.model_params['num_leaves'],
            "feature_fraction": self.model_params['feature_fraction'],
            "bagging_fraction": self.model_params['bagging_fraction'],
            "bagging_freq": self.model_params['bagging_freq'],
            "min_child_samples": self.model_params['min_child_samples']
        }
        gbm = lgb.train(param, dtrain)
        return gbm
    
    def train(self):
        model_knn, neighbors, prediction = self.knn()
        model_lgb = self.lightgbm(neighbors, prediction)
        return model_lgb, model_knn
    
    def predict(self,age_study,model_lgb, model_knn):
        neighbors = pd.DataFrame(model_knn.kneighbors(return_distance=False))
        y_pred_knn = pd.DataFrame(model_knn.predict(self.X_pred))
        X_pred = self.data(self.X_pred, neighbors, y_pred_knn)
        y_pred_lgb = pd.DataFrame(model_lgb.predict(X_pred[0]))

        return y_pred_knn.values.tolist(), y_pred_lgb.values.tolist(), self.fit, X_pred[0]['customer_id']
    
if __name__ == "__main__":
    predictions = []
    y_pred = []
    r = []
    customer_ids = []
    for each in best_models:
        print(each)
        young_age = each[1]
        old_age = young_age+each[2]
        print(old_age)
        age_study = X.loc[(X['age'] >= young_age) & (X['age'] <= old_age)]
        
        t = train(age_study,each[0])
        model_lgb, model_knn = t.train()
        y_pred_knn, y_pred_lgb, encoder, customer_id = t.predict(age_study, model_lgb, model_knn)
        i = 0
        
        print(f'age_length {len(age_study)}')
        print(f'knn_ prediction {len(y_pred_knn)}')
        print(f'lgb_ prediction {len(y_pred_lgb)}')
        print(f'customer_id {len(customer_id)}')
        print(age_study.columns)
        customer_id = customer_id.to_list()
        while i < len(y_pred_knn):
            k = round(y_pred_lgb[i][0])
            m = encoder.inverse_transform([round(y_pred_knn[i][0])])
            
            if k == 1:
                r.append('1')
            else:
                r.append(str(m[0]))
            i+=1
        customer_ids.extend(customer_id)

[{'neighbors': 3, 'num_leaves': 173, 'feature_fraction': 0.5186323020532635, 'bagging_fraction': 0.6163552066220521, 'bagging_freq': 6, 'min_child_samples': 31}, 16, 5, None]
21

d:\soprisanalytics\kaggle\dtsa-5509-supervised-learning-final-project\venv\lib\site-packages\pandas\core\indexing.py:1667: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

[0]
[LightGBM] [Info] Number of positive: 56415, number of negative: 9151
[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.002873 seconds.
You can set `force_row_wise=true` to remove the overhead.
And if memory is not enough, you can set `force_col_wise=true`.
[LightGBM] [Info] Total Bins 2618
[LightGBM] [Info] Number of data points in the train set: 65566, number of used features: 14
[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.860431 -> initscore=1.818872
[LightGBM] [Info] Start training from score 1.818872
age_length 65566
knn_ prediction 65566
lgb_ prediction 65566
customer_id 65566
Index(['postal_codes', 'fashion_news_frequency', 'club_member_status',
       'customer_id', 'age', '0_week', '1_week', '2_week', '3_week', '4_week',
       '5_week', '6_week', '7_week', '8_week', '9_week', '0_weeks'],
      dtype='object')
[{'neighbors': 2, 'num_leaves': 115, 'feature_fraction': 0.8414673863714155, 'bagging_fraction': 0.5192146843669733, 'bagging_freq': 7, 'min_child_samples': 69}, 21, 5, None]
26

d:\soprisanalytics\kaggle\dtsa-5509-supervised-learning-final-project\venv\lib\site-packages\pandas\core\indexing.py:1667: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

[0]
[LightGBM] [Info] Number of positive: 104186, number of negative: 19332
[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.005148 seconds.
You can set `force_row_wise=true` to remove the overhead.
And if memory is not enough, you can set `force_col_wise=true`.
[LightGBM] [Info] Total Bins 2820
[LightGBM] [Info] Number of data points in the train set: 123518, number of used features: 14
[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.843488 -> initscore=1.684416
[LightGBM] [Info] Start training from score 1.684416
age_length 123518
knn_ prediction 123518
lgb_ prediction 123518
customer_id 123518
Index(['postal_codes', 'fashion_news_frequency', 'club_member_status',
       'customer_id', 'age', '0_week', '1_week', '2_week', '3_week', '4_week',
       '5_week', '6_week', '7_week', '8_week', '9_week', '0_weeks'],
      dtype='object')
[{'neighbors': 2, 'num_leaves': 130, 'feature_fraction': 0.7006697050976558, 'bagging_fraction': 0.9226721145541669, 'bagging_freq': 5, 'min_child_samples': 98}, 26, 5, None]
31

d:\soprisanalytics\kaggle\dtsa-5509-supervised-learning-final-project\venv\lib\site-packages\pandas\core\indexing.py:1667: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

[0]
[LightGBM] [Info] Number of positive: 56660, number of negative: 10478
[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.002935 seconds.
You can set `force_row_wise=true` to remove the overhead.
And if memory is not enough, you can set `force_col_wise=true`.
[LightGBM] [Info] Total Bins 2710
[LightGBM] [Info] Number of data points in the train set: 67138, number of used features: 14
[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.843933 -> initscore=1.687791
[LightGBM] [Info] Start training from score 1.687791
age_length 67138
knn_ prediction 67138
lgb_ prediction 67138
customer_id 67138
Index(['postal_codes', 'fashion_news_frequency', 'club_member_status',
       'customer_id', 'age', '0_week', '1_week', '2_week', '3_week', '4_week',
       '5_week', '6_week', '7_week', '8_week', '9_week', '0_weeks'],
      dtype='object')


print(len(customer_ids))
print(len(r))
d = {'Customer_Ids':customer_ids,'Prediction':r}
df = pd.DataFrame(d)
df['Customer_Ids'] = customer_id_encoding.inverse_transform(df['Customer_Ids'])

print(df)

256222
256222
                                             Customer_Ids Prediction
0       588e32d81a3c5eda8439b88654cb2f7acb336c9e028ca7...          1
1       5374130b2d260391c22de34dd9237c9c5bcc320b1fec31...          1
2       7beee2baccfda3501868a56642b738bc52bcf1804d4612...          1
3       f0d47c078e1ae6a6c3552a8bd7c7c4226db650136b487a...          1
4       fd90cc5dabce374693d54684cc8dba552f2a40c9a2619f...  762846027
...                                                   ...        ...
256217  5a97b2a5f70bc5f421a3f9a41067985a2c9dd1fa6545da...          1
256218  3048faedf0a20e13a89e93e3e4d33b325ad9f7cd625933...          1
256219  6984408761c01ab4db16ecb7b4b934b6a5958853033bc9...          1
256220  672b61c9c5af04c8e38e98214433c6a9f4dd1386c2b5aa...          1
256221  c6cc3dc177527df77d43647414aa12938ebe140e9c77a4...          1

[256222 rows x 2 columns]

	index	customer_id	club_member_status	fashion_news_frequency	age	postal_code
0	0	0001d44dbe7f6c4b35200abdb052c77a87596fe1bdcc37...	ACTIVE	Regularly	44.0	930b19ae7db8abb5a27f4da10217755a7305b4c452f5e0...
1	66	0008d644deb96bdc0ca262f161cf6d5e9a4e619bb75faa...	ACTIVE	NONE	32.0	7d1a68652e11ef1653149e002e3508dabfe095f4cd93c4...
2	70	000da6daeb90ef9a70238bf9b1aa54c7ce40a5e0fcf220...	ACTIVE	Regularly	42.0	a67d8a85fb9cd7728a4a7a02244328b27bdb04e93a7b21...
3	82	000fb6e772c5d0023892065e659963da90b1866035558e...	ACTIVE	Regularly	42.0	68ca4d9d6051d9c10b917d36bf9cb4afbadc551f7e4feb...
4	140	00194061f3caa80bf10d615bf406bc5959a3bd799e4f21...	ACTIVE	NONE	24.0	b8d066ae8df3d645c85a5ba869e2fa7b528cf27d92dc06...

	article_id
bucket	0.0	1.0	2.0	3.0	4.0	5.0	6.0	7.0	8.0	9.0
customer_id
00000dbacae5abe5e23885899a1fa44253a17956c6d1c3d25f88aa139fdfc657	NaN	NaN	NaN	NaN	NaN	NaN	NaN	568601043	NaN	NaN
000058a12d5b43e67d225668fa1f8d618c13dc232df0cad8ffe7ad4a1091e318	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	794321007	NaN
00006413d8573cd20ed7128e53b7b13819fe5cfc2d801fe7fc0f26dd8d65a85a	NaN	NaN	NaN	NaN	896152002 730683050 927530004 791587015	NaN	NaN	NaN	NaN	NaN
0000757967448a6cb83efb3ea7a3fb9d418ac7adf2379d8cd0c725276a467a2a	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	719530003 448509014	NaN
0000945f66de1a11d9447609b8b41b1bc987ba185a5496ae8831e8493afa24ff	760084003 760084013	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN

	customer_id	club_member_status	fashion_news_frequency	age	postal_code	(article_id, 0.0)	(article_id, 1.0)	(article_id, 2.0)	(article_id, 3.0)	(article_id, 4.0)	(article_id, 5.0)	(article_id, 6.0)	(article_id, 7.0)	(article_id, 8.0)	(article_id, 9.0)
4	00194061f3caa80bf10d615bf406bc5959a3bd799e4f21...	ACTIVE	NONE	24.0	b8d066ae8df3d645c85a5ba869e2fa7b528cf27d92dc06...	796535007 759482003	NaN	NaN	NaN	NaN	NaN	NaN	NaN	572998005	NaN
5	001cd4541544c87a2c9fb19bad430646eadc24226d8358...	ACTIVE	NONE	26.0	27c895023a1581925eabd7adbb36810765c018fa818651...	897693002	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
6	001ea4e9c54f7e9c88811260d954edc059d596147e1cf8...	ACTIVE	NONE	25.0	63c82aec4da25b6b5f967abe4e17d112b24e53643e916d...	887770004 878941004	NaN	NaN	NaN	882899012 860949002 739590032	NaN	NaN	NaN	NaN	863646005
7	001eacd7b51c28d306e48b93b28d5f209a1da073bb94c7...	ACTIVE	NONE	27.0	e1474f9a871813651183c7fc5555a9d5d83106a5818528...	859105008	886241003 852746001 817472007 914966002 900659...	902694001 881933001	837741001 850259002	NaN	NaN	NaN	NaN	NaN	NaN
11	002efd1ae90ecbd94196276378da2ac5ff49fe79ee13c2...	ACTIVE	Regularly	21.0	7e42dc67e7fea94838531e8d556c0c21ba19365b61ae61...	833530005 806241007 833499005 833499005 941005...	NaN	NaN	NaN	NaN	856270003 856270003 918292011 918292011 827968...	NaN	935689001 935689001 863583002 935092001 920084...	NaN	NaN

	customer_id	club_member_status	fashion_news_frequency	age	postal_code	0_week	1_week	2_week	3_week	4_week	5_week	6_week	7_week	8_week	9_week
294244	588e32d81a3c5eda8439b88654cb2f7acb336c9e028ca7...	ACTIVE	NONE	16.0	c49603b16392d7701f70a9eee862d321eb171d3a18e4d8...	1	1	1	1	1	924142001	1	1	1	1
260706	5374130b2d260391c22de34dd9237c9c5bcc320b1fec31...	ACTIVE	NONE	16.0	9f083c65954fb1ceb1cfd74d2e6dba70428d9005743084...	1	1	1	1	806388028 916468003 806388018	1	1	1	806388002	1
365141	7beee2baccfda3501868a56642b738bc52bcf1804d4612...	ACTIVE	Regularly	16.0	3f2c72cff97fb4cd9e85a5f351f4fc4dc2206f99aad8f8...	1	1	1	1	1	1	1	456163085 926502001 809961002	1	1
223433	f0d47c078e1ae6a6c3552a8bd7c7c4226db650136b487a...	ACTIVE	NONE	16.0	b8572c20d2610a46bbdd304058a19a7dec301a892424e2...	1	1	1	850426003 855793002	1	1	1	801673001 904424001 911214001	1	1
199856	fd90cc5dabce374693d54684cc8dba552f2a40c9a2619f...	ACTIVE	NONE	16.0	f212e80f468490c6f135da6c0910d01e19a420d7deb4c8...	1	1	910199002	1	1	1	1	1	1	1

Summary¶

Data¶

Data Strategy¶

Prediction Strategy¶

Conclusion and Next steps¶

Gitlab¶

Data Ingestion¶

Feature Creation¶

EDA part 1¶

Data Munging Part 1¶

Data Munging Part 2¶

EDA Part 2¶

Model Optimization¶

Optimization Graphs¶

Training and Prediction¶